package lumis.content.service.serialization;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.hibernate.Hibernate;
import org.w3c.dom.Node;

import lumis.content.ContentManagerFactory;
import lumis.content.service.ContentService;
import lumis.doui.DouiManagerFactory;
import lumis.doui.search.common.DouiIndexerFactory;
import lumis.portal.PortalException;
import lumis.portal.PortalObjectNotFoundException;
import lumis.portal.UnexpectedException;
import lumis.portal.PortalContext.RelativePathType;
import lumis.portal.dao.jdbc.ITransactionJdbc;
import lumis.portal.manager.ManagerFactory;
import lumis.portal.manager.ManagerFactoryInternal;
import lumis.portal.progress.ProgressConfig;
import lumis.portal.serialization.FileCopyTransactionObserver;
import lumis.portal.serialization.SerializationException;
import lumis.portal.service.IService;
import lumis.portal.service.ServiceConfig;
import lumis.portal.serviceinstance.ServiceInstanceConfig;
import lumis.portal.serviceinstance.serialization.ServiceInstanceBaseSerializationContext;
import lumis.portal.serviceinstance.serialization.ServiceInstanceDeserializationContext;
import lumis.portal.serviceinstance.serialization.ServiceInstanceSerializationContext;
import lumis.portal.transaction.AbstractTransactionObserver;
import lumis.portal.transaction.ITransactionObserver;
import lumis.portal.transaction.PortalTransactionFactory;
import lumis.util.FileUtil;
import lumis.util.ITransaction;
import lumis.util.XmlUtil;

/**
 * Manager for Content Service serialization
 * 
 * @version $Revision: 12710 $ $Date: 2011-03-18 17:00:16 -0300 (Fri, 18 Mar 2011) $
 * @since 4.0.8
 */
public class ContentServiceSerializationManager implements IContentServiceSerializationManager
{
	private static class ContentSerializableConfig
	{
		List<ContentSourceSerializableConfig> sources = new ArrayList<ContentSourceSerializableConfig>();
		int dataCount = 0;
	}
	
	private static class ContentSourceSerializableConfig
	{
		public String id;
		public TableSerializableConfig tableData = null;
		public String sourceFromStatement;
	}
	
	private static class TableSerializableConfig
	{
		public String tableName;
		public List<TableColumnSerializableConfig> columnNames = new ArrayList<TableColumnSerializableConfig>();
		public int numDataFiles = 0;
		public CachedTemporaryDataSerializableConfig cachedData = new CachedTemporaryDataSerializableConfig();
	}
	
	private static class CachedTemporaryDataSerializableConfig
	{
		public List<TableRowDataSerializableConfig> cachedTemporaryData = new ArrayList<TableRowDataSerializableConfig>();
		public int cachedTemporaryDataSize = 0;
		
		public void resetCachedTemporaryData()
		{
			cachedTemporaryData = new ArrayList<TableRowDataSerializableConfig>();;
			cachedTemporaryDataSize = 0;
		}
		
		public void addRow(TableRowDataSerializableConfig row)
		{
			cachedTemporaryData.add(row);
			for (Object value : row.rowData)
			{
				if (value != null)
					cachedTemporaryDataSize += value.toString().length();
			}
		}
	}
	
	private static class TableRowDataSerializableConfig
	{
		public List<Object> rowData = new ArrayList<Object>();
	}
	
	private static class TableColumnSerializableConfig
	{
		public String columnName;
		public boolean isClob = false;
	}
	
	private static class FileTableSerializableConfig extends TableSerializableConfig
	{
		public List<String> fileNames = new ArrayList<String>();
	}
	
	private static class ColumnValue
	{
		public Object value;
		public boolean isClob;
		
		public ColumnValue(Object value, boolean isClob)
		{
			this.value = value;
			this.isClob = isClob;
		}
	}
	
	private static final String configPath = "ContentService/structure.xml";
	private static final String dataFileDirectoryPath = "ContentService";
	private static final String dataFilePath = dataFileDirectoryPath+"/structure.xml";
	
	
	// SERIALIZATION
	public void serialize(ServiceInstanceSerializationContext serviceInstanceSerializationContext) throws PortalException
	{
		// setup serialization settings
		setupSerializationContext(serviceInstanceSerializationContext);

		// build structure object
		ContentServiceInstanceSerializableConfig contentServiceInstanceSerializableConfig = new ContentServiceInstanceSerializableConfig();

		// serialize workflow, if necessary
		contentServiceInstanceSerializableConfig.setWorkflow(ContentManagerFactory.getWorkflowSerializationManager().serialize(serviceInstanceSerializationContext));

		// serialize structure object
		serviceInstanceSerializationContext.logDebug("Serializing content service instance structure object");
		ManagerFactoryInternal.getSerializationManager2().serializeObject(contentServiceInstanceSerializableConfig, serviceInstanceSerializationContext.getStructureDirectory(), configPath, "contentService",
				serviceInstanceSerializationContext.getSerializationSettings());

		if (serviceInstanceSerializationContext.getIncludeContent())
		{
			// serialize the content data
			serviceInstanceSerializationContext.logDebug("Serializing content service instance data");

			ContentSerializableConfig contentConfig = new ContentSerializableConfig();
			
			serializeData(serviceInstanceSerializationContext, contentConfig);
	
			if (contentConfig.sources.size() > 0)
			{
				ManagerFactoryInternal.getSerializationManager2().serializeObject(contentConfig, serviceInstanceSerializationContext.getDataDirectory(), dataFilePath, null,
						serviceInstanceSerializationContext.getSerializationSettings());
			}
		}
	}

	/**
	 * Serializes the content data to a the given StringBuilder.
	 * <p>
	 * This method parses the douidefinition of the given content service and for each 
	 * source that is of type 'contentTable' and that has a 'table' tag within it, it 
	 * reads the table data with all the related metadata and puts this information in
	 * an xml string stored in the contentData variable passed in.
	 * <p>
	 * If any source has field data types of file or files, the respective files are 
	 * also included in the exported data.
	 * 
	 * @param serviceInstanceSerializationContext
	 * @param contentData
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void serializeData(ServiceInstanceSerializationContext serviceInstanceSerializationContext, ContentSerializableConfig contentConfig) throws PortalException
	{
		ServiceInstanceConfig serviceInstanceConfig = ManagerFactory.getServiceInstanceManager().get(serviceInstanceSerializationContext.getServiceInstanceId(), serviceInstanceSerializationContext.getTransaction());
		String serviceId = serviceInstanceConfig.getServiceId();
		ServiceConfig serviceConfig = ManagerFactory.getServiceManager().get(serviceInstanceSerializationContext.getSessionConfig(), serviceId, serviceInstanceSerializationContext.getTransaction());
		String serviceInstanceName = ManagerFactory.getLocalizationManager().localize(serviceInstanceConfig.getName(), serviceConfig.getResource());
		Node serviceDefinitionNode = DouiManagerFactory.getDouiManager().getDouiServiceDefinitionNode(serviceId, serviceInstanceSerializationContext.getTransaction());

		serializeAllContentMetadata(serviceInstanceSerializationContext, contentConfig, serviceInstanceConfig.getId());
		serializeAllFileData(serviceInstanceSerializationContext, contentConfig, serviceInstanceConfig.getId());
		
		// initialize progress bar
		//serviceInstanceSerializationContext.initializeInnerProgressBar(100, "STR_EXPORTING_SERVICE_INSTANCE_DATA;" + serviceInstanceName);
		
		ArrayList<String> tableNames = new ArrayList<String>();
		
		Node[] sourceNodes = XmlUtil.selectNodes("service/sources/source[@type='contentTable']", serviceDefinitionNode);
		for (Node sourceNode : sourceNodes)
		{
			Node tableNode = XmlUtil.selectSingleNode("table", sourceNode);
			if (tableNode != null)
			{
				String tableName = XmlUtil.getTextContent(tableNode);

				if (tableNames.contains(tableName))
					continue;

				String sourceId = XmlUtil.readAttributeString("id", sourceNode);
				
				// get primaryKeyFieldColumnName
				String primaryKeyFieldColumnName;
				Node primaryKeyFieldNode = XmlUtil.selectSingleNode("fields/field[@isPrimaryKey='true']", sourceNode);
				if (primaryKeyFieldNode != null)
				{
					primaryKeyFieldColumnName = XmlUtil.readAttributeString("columnName", primaryKeyFieldNode);
					if (primaryKeyFieldColumnName == null)
						primaryKeyFieldColumnName = XmlUtil.readAttributeString("id", primaryKeyFieldNode);
				}
				else
				{
					throw new PortalException("primary key field not found for source with id: "+sourceId);
				}
				
				// serialize table
				String sourceFromStatement = tableName + " inner join lum_ContentVersion on " +
						tableName + "." + primaryKeyFieldColumnName + 
						"=lum_ContentVersion.itemId inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
				String query = "select " + tableName + ".* from " + sourceFromStatement;
				
				ContentSourceSerializableConfig source = new ContentSourceSerializableConfig();
				source.id = sourceId;
				source.sourceFromStatement = sourceFromStatement;
				source.tableData = new TableSerializableConfig();
				source.tableData.tableName = tableName;
				contentConfig.sources.add(source);
				
				// serialize all files related to the source.
				
				serializeQueryResults(serviceInstanceSerializationContext, 
						source.id, source.tableData, query, serviceInstanceConfig.getId(), 
						serviceInstanceSerializationContext.getTransaction());
				
				tableNames.add(tableName);
			}
		}
		
		// now make sure that any simple tables that have relations to content are also exported.
		if(tableNames.size() > 0)
		{
			sourceNodes = XmlUtil.selectNodes("service/sources/source[@type='table']", serviceDefinitionNode);
			for (Node sourceNode : sourceNodes)
			{
				String sourceId = XmlUtil.readAttributeString("id", sourceNode);
				
				Node tableNode = XmlUtil.selectSingleNode("table", sourceNode);
				if (tableNode == null)
					continue;
				
				String tableName = XmlUtil.getTextContent(tableNode);

				if (tableNames.contains(tableName))
					continue;
				
				Node relationsNode = XmlUtil.selectSingleNode("relations", sourceNode);
				if(relationsNode == null)
					continue;
				
				Node[] relationNodes = XmlUtil.selectNodes("relation", relationsNode);
				if(relationNodes.length == 0)
					continue;
				
				ContentSourceSerializableConfig relatedSource = null;
				Node sourceRelationNode = null;
				for(Node relationNode : relationNodes)
				{
					String cascadeOnDelete = XmlUtil.readAttributeString("cascadeOnDelete", relationNode);
					if(cascadeOnDelete == null || !cascadeOnDelete.equals("true"))
						continue;
					String foreignSourceId = XmlUtil.readAttributeString("foreignSourceId", relationNode);
					if(foreignSourceId == null)
						foreignSourceId = XmlUtil.readAttributeString("id", relationNode);

					// check if the foreign source is in the list of processed sources
					for(ContentSourceSerializableConfig processedSource : contentConfig.sources)
					{
						if(processedSource.id.equals(foreignSourceId))
						{
							relatedSource = processedSource;
							sourceRelationNode = relationNode;
							break;
						}
					}
					
					if(relatedSource != null)
						break;
				}
				
				if(relatedSource == null)
					continue;
				
				if(relatedSource.tableData == null)
					continue;
				
				Node relatedSourceNode = XmlUtil.selectSingleNode("source[@id='" + 
						relatedSource.id + "']", sourceNode.getParentNode());
				if (relatedSourceNode == null)
					continue;
				
				Node[] relationFields = XmlUtil.selectNodes("relationField", sourceRelationNode);
				if(relationFields.length == 0)
					continue;
				
				int relationFieldsCount = 0;
				StringBuilder sourceFromStatement = new StringBuilder(200 + relatedSource.sourceFromStatement.length());
				sourceFromStatement.append(relatedSource.sourceFromStatement +
						" inner join " + tableName + " on ");
				for(Node relationField : relationFields)
				{
					// generate SQL statement this relationField
					String fieldId = XmlUtil.readAttributeString("fieldId", relationField);
					String foreignFieldId = XmlUtil.readAttributeString("foreignFieldId", relationField);
					Node foreignFieldNode = XmlUtil.selectSingleNode("fields/field[@id='" + foreignFieldId + "']", relatedSourceNode);
					
					if (foreignFieldNode != null)
					{
						if(relationFieldsCount > 1)
							sourceFromStatement.append(" and ");
						
						if (XmlUtil.readAttributeBoolean("isContentId", foreignFieldNode, false))
						{
							sourceFromStatement.append(tableName + "." + fieldId + " = lum_Content.id");
						}
						else
						{
							sourceFromStatement.append(tableName + "." + fieldId + 
									" = " + relatedSource.tableData.tableName + "." + foreignFieldId);
						}
						
						relationFieldsCount++;
					}
				}
				
				String query = "select " + tableName + ".* from " + sourceFromStatement;

				ContentSourceSerializableConfig source = new ContentSourceSerializableConfig();
				source.id = sourceId;
				source.sourceFromStatement = sourceFromStatement.toString();
				source.tableData = new TableSerializableConfig();
				source.tableData.tableName = tableName;
				contentConfig.sources.add(source);
					
				serializeQueryResults(serviceInstanceSerializationContext, source.id, source.tableData, 
						query, serviceInstanceConfig.getId(), serviceInstanceSerializationContext.getTransaction());
				
				tableNames.add(tableName);
			}
		}
		
		// finalize progress bar
		serviceInstanceSerializationContext.finalizeInnerProgressBar("STR_EXPORTING_SERVICE_INSTANCE_DATA_FINISHED;" + serviceInstanceName);
	}
	
	protected void serializeAllFileData(ServiceInstanceSerializationContext serviceInstanceSerializationContext, ContentSerializableConfig contentConfig, String serviceInstanceId) throws PortalException
	{
		ITransaction transaction = serviceInstanceSerializationContext.getTransaction();

		// serialize table: lum_File
		String query = "select lum_File.* from lum_File where serviceInstanceId=?";
		ContentSourceSerializableConfig source = new ContentSourceSerializableConfig();
		source.id = "lum_File";
		source.tableData = new FileTableSerializableConfig();
		source.tableData.tableName = "lum_File";
		contentConfig.sources.add(source);
		
		serializeQueryResults(serviceInstanceSerializationContext, "lum_File", source.tableData, query, serviceInstanceId, transaction);
		serializeAttachedFiles(serviceInstanceSerializationContext, (FileTableSerializableConfig)source.tableData);
		
		// serialize table: lum_Files
		query = "select distinct lum_Files.* from lum_Files inner join lum_FilesList on lum_Files.filesId=lum_FilesList.filesId inner join lum_File on lum_FilesList.fileId=lum_File.fileId and lum_File.serviceInstanceId=?";
		serializeContentMetaData(query, "lum_Files", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);

		// serialize table: lum_FilesList
		query = "select lum_FilesList.* from lum_FilesList inner join lum_File on lum_FilesList.fileId=lum_File.fileId and lum_File.serviceInstanceId=?";
		serializeContentMetaData(query, "lum_FilesList", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
	}
	
	protected void serializeAllContentMetadata(ServiceInstanceSerializationContext serviceInstanceSerializationContext, ContentSerializableConfig contentConfig, String serviceInstanceId) throws PortalException
	{
		ITransaction transaction = serviceInstanceSerializationContext.getTransaction();
		
		// serialize table: lum_Content
		String query = "select lum_Content.* from lum_Content where serviceInstanceId=?";
		serializeContentMetaData(query, "lum_Content", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);

		// serialize table: lum_ContentLocale
		query = "select lum_ContentLocale.* from lum_ContentLocale inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentLocale", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);

		// serialize table: lum_ContentVersion
		query = "select lum_ContentVersion.* from lum_ContentVersion inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentVersion", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);

		// serialize table: lum_ContentPublication
		query = "select lum_ContentPublication.* from lum_ContentPublication inner join lum_ContentVersion on lum_ContentPublication.contentVersionId=lum_ContentVersion.id inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentPublication", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
		
		// serialize table: lum_ContentPubPrincipal
		query = "select lum_ContentPubPrincipal.* from lum_ContentPubPrincipal inner join lum_ContentPublication on lum_ContentPubPrincipal.contentPublicationId=lum_ContentPublication.id inner join lum_ContentVersion on lum_ContentPublication.contentVersionId=lum_ContentVersion.id inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentPubPrincipal", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
		
		// serialize table: lum_ContentPubServiceInst
		query = "select lum_ContentPubServiceInst.* from lum_ContentPubServiceInst inner join lum_ContentPublication on lum_ContentPubServiceInst.contentPublicationId=lum_ContentPublication.id inner join lum_ContentVersion on lum_ContentPublication.contentVersionId=lum_ContentVersion.id inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentPubServiceInst", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
		
		// serialize table: lum_ContentComment
		query = "select lum_ContentComment.* from lum_ContentComment inner join lum_ContentLocale on lum_ContentComment.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentComment", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);

		// serialize table: lum_ContentAssociation
		query = "select lum_ContentAssociation.* from lum_ContentAssociation inner join lum_ContentVersion on lum_ContentAssociation.contentVersionId=lum_ContentVersion.id inner join lum_ContentLocale on lum_ContentVersion.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_ContentAssociation", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
		
		// serialize table: lum_WorkflowMetaData
		query = "select lum_WorkflowMetaData.* from lum_WorkflowMetaData inner join lum_ContentLocale on lum_WorkflowMetaData.contentLocaleId=lum_ContentLocale.id inner join lum_Content on lum_Content.serviceInstanceId=? and lum_ContentLocale.contentId=lum_Content.id";
		serializeContentMetaData(query, "lum_WorkflowMetaData", serviceInstanceSerializationContext, contentConfig, serviceInstanceId, transaction);
	}
	
	protected void serializeContentMetaData(String query, String tableName, ServiceInstanceSerializationContext serviceInstanceSerializationContext, ContentSerializableConfig contentConfig, String serviceInstanceId, ITransaction transaction) throws PortalException
	{
		ContentSourceSerializableConfig source = new ContentSourceSerializableConfig();
		source.id = tableName;
		source.tableData = new TableSerializableConfig();
		source.tableData.tableName = tableName;
		contentConfig.sources.add(source);
		
		serializeQueryResults(serviceInstanceSerializationContext, source.id, source.tableData, query, serviceInstanceId, transaction);
	}
	

	/**
	 * Executes and serializes the query results into the given StringBuilder
	 * @param query
	 * @param filterFieldValue
	 * @param transaction
	 * @param queryResultsData
	 * @param tableName
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void serializeQueryResults(ServiceInstanceSerializationContext serviceInstanceSerializationContext, String sourceId, TableSerializableConfig tableSerializedData, String query, String filterFieldValue, ITransaction transaction) throws PortalException
	{
		try
		{
			ITransactionJdbc daoTransactionJdbc = (ITransactionJdbc) transaction;
			Connection connection = daoTransactionJdbc.getConnection();

			PreparedStatement statement = connection.prepareStatement(query.toString());
			try
			{
				statement.setString(1, filterFieldValue);
				ResultSet resultSet = statement.executeQuery();
				try
				{
					while (resultSet.next())
					{
						if (tableSerializedData.columnNames.size() == 0)
						{
							for (int curColumn = 1; curColumn <= resultSet.getMetaData().getColumnCount(); curColumn++)
							{
								TableColumnSerializableConfig curColumnConfig = new TableColumnSerializableConfig();
								curColumnConfig.columnName = resultSet.getMetaData().getColumnName(curColumn);
								int curColType = resultSet.getMetaData().getColumnType(curColumn);
								if (curColType == Types.CLOB || curColType == Types.LONGVARCHAR)
									curColumnConfig.isClob = true;
								
								tableSerializedData.columnNames.add(curColumnConfig);
							}
						}
						
						TableRowDataSerializableConfig curRow = new TableRowDataSerializableConfig();
						for (int curColumn = 1; curColumn <= resultSet.getMetaData().getColumnCount(); curColumn++)
						{
							Object dataValue;
							int curColType = resultSet.getMetaData().getColumnType(curColumn);
							if (curColType == Types.TIMESTAMP || curColType == Types.DATE)
							{
								try
								{
									dataValue = resultSet.getTimestamp(curColumn);
								}
								catch (SQLException e)
								{
									dataValue = resultSet.getObject(curColumn);
								}
							}
							else
							{
								TableColumnSerializableConfig tableColumn = tableSerializedData.columnNames.get(curColumn-1);
								if (tableColumn.isClob)
									dataValue = Hibernate.TEXT.nullSafeGet(resultSet, tableColumn.columnName);
								else
									dataValue = resultSet.getObject(curColumn);
							}

							curRow.rowData.add(dataValue);
							
							// if saving file path info, include this in the structure as well for later serialization of file
							if (dataValue != null && tableSerializedData instanceof FileTableSerializableConfig && resultSet.getMetaData().getColumnName(curColumn).equalsIgnoreCase("fullpath"))
							{
								FileTableSerializableConfig fileResultsStructure = (FileTableSerializableConfig) tableSerializedData;
								fileResultsStructure.fileNames.add(dataValue.toString());
							}
						}
						tableSerializedData.cachedData.addRow(curRow);
						
						
						if (tableSerializedData.cachedData.cachedTemporaryDataSize > 100000)
						{
							SaveResultsDataBlock(tableSerializedData, serviceInstanceSerializationContext, sourceId);
						}
					}

					SaveResultsDataBlock(tableSerializedData, serviceInstanceSerializationContext, sourceId);
				}
				finally
				{
					resultSet.close();
				}
			}
			finally
			{
				statement.close();
			}
		}
		catch (SQLException e)
		{
			throw new UnexpectedException(e);
		}
	}

	private void SaveResultsDataBlock(TableSerializableConfig tableData, ServiceInstanceSerializationContext serviceInstanceSerializationContext, String sourceId) throws PortalException
	{
		if (tableData.cachedData.cachedTemporaryDataSize > 0)
		{
			tableData.numDataFiles++;
			ManagerFactoryInternal.getSerializationManager2().serializeObject(tableData.cachedData, serviceInstanceSerializationContext.getDataDirectory(), dataFileDirectoryPath+"/"+sourceId+"/"+tableData.tableName+"/"+tableData.numDataFiles+".xml", null,
					serviceInstanceSerializationContext.getSerializationSettings());
		
			tableData.cachedData.resetCachedTemporaryData();
		}
	}
	
	/**
	 * Serializes all referenced files in the variable filesXmlString into the serialization context.
	 * 
	 * @param serializationContext
	 * @param filesXmlString
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void serializeAttachedFiles(ServiceInstanceSerializationContext serializationContext, FileTableSerializableConfig fileData) throws PortalException
	{
		for (String filePath : fileData.fileNames)
			serializeAttachedFile(serializationContext, filePath);
	}

	/**
	 * Serializes the given file into the serialization context.
	 * 
	 * @param serializationContext
	 * @param filePath
	 * @throws SerializationException
	 * @since 4.0.11
	 */
	protected void serializeAttachedFile(ServiceInstanceSerializationContext serializationContext, String filePath) throws SerializationException
	{
		File srcFile = new File(RelativePathType.Data.getAbsolutePath(filePath));
		InputStream inputStream = null;
		OutputStream outputStream = null;
		try
		{
			inputStream = new FileInputStream(srcFile);
			outputStream = serializationContext.getDataDirectory().getOutputStream(dataFileDirectoryPath+"/lum_files"+filePath);
			FileUtil.copyStream(inputStream, outputStream);
		}
		catch (IOException e)
		{
			throw new SerializationException("STR_ERROR_SERIALIZING_FILE;" + filePath, e);
		}
		finally
		{
			try
			{
				if (inputStream != null)
					inputStream.close();
			}
			catch (IOException e)
			{
				serializationContext.logDebug("Error closing file copy inputStream", e);
			}
			try
			{
				if (outputStream != null)
					outputStream.close();
			}
			catch (IOException e)
			{
				serializationContext.logDebug("Error closing file copy outputStream", e);
			}
		}
	}
	
	// DESERIALIZATION

	public void deserialize(ServiceInstanceDeserializationContext serviceInstanceDeserializationContext) throws PortalException
	{
		// setup serialization settings
		setupSerializationContext(serviceInstanceDeserializationContext);

		// deserialize structure object
		serviceInstanceDeserializationContext.logDebug("Deserializing content service instance structure object");
		ContentServiceInstanceSerializableConfig contentServiceInstanceSerializableConfig = ManagerFactoryInternal.getSerializationManager2().deserializeObject(ContentServiceInstanceSerializableConfig.class,
				serviceInstanceDeserializationContext.getStructureDirectory(), configPath, "contentService", serviceInstanceDeserializationContext.getSerializationSettings(), true);

		// backward-compatibility: if there is not a serialized contentserviceinstance, skip it
		if (contentServiceInstanceSerializableConfig == null)
			return;

		// deserialize workflow
		ContentManagerFactory.getWorkflowSerializationManager().deserialize(serviceInstanceDeserializationContext, contentServiceInstanceSerializableConfig.getWorkflow());
	}

	public void deserializeContent(ServiceInstanceDeserializationContext serviceInstanceDeserializationContext) throws PortalException
	{
		// setup serialization settings
		setupSerializationContext(serviceInstanceDeserializationContext);

		if (serviceInstanceDeserializationContext.getIncludeContent())
		{			
			// read the content data
			ContentSerializableConfig contentData = ManagerFactoryInternal.getSerializationManager2().deserializeObject(ContentSerializableConfig.class, serviceInstanceDeserializationContext.getDataDirectory(), dataFilePath, null,
					serviceInstanceDeserializationContext.getSerializationSettings(), true);
	
			if (contentData != null)
			{
				// first clear all the existing content data related to this service instance
				ServiceInstanceConfig serviceInstanceConfig = ManagerFactory.getServiceInstanceManager().get(serviceInstanceDeserializationContext.getServiceInstanceId(), serviceInstanceDeserializationContext.getTransaction());
				IService service = ManagerFactory.getServiceManager().getService(serviceInstanceDeserializationContext.getSessionConfig(), serviceInstanceConfig.getServiceId(), serviceInstanceDeserializationContext.getTransaction());
	
				if (service instanceof ContentService)
					((ContentService) service).deleteContent(serviceInstanceDeserializationContext.getSessionConfig(), serviceInstanceConfig, serviceInstanceDeserializationContext.getTransaction());
	
				// next load new content data
				serviceInstanceDeserializationContext.logDebug("Deserializing content service instance data");
				deserializeData(contentData, serviceInstanceDeserializationContext);
			}
		}
	}

	/**
	 * Deserializes the given content into the respective tables.
	 * 
	 * @param contentData
	 * @param serviceInstanceDeserializationContext
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void deserializeData(final ContentSerializableConfig contentData, final ServiceInstanceDeserializationContext serviceInstanceDeserializationContext) throws PortalException
	{
		final ServiceInstanceConfig serviceInstanceConfig = ManagerFactory.getServiceInstanceManager().get(serviceInstanceDeserializationContext.getServiceInstanceId(), serviceInstanceDeserializationContext.getTransaction());
		ServiceConfig serviceConfig = ManagerFactory.getServiceManager().get(serviceInstanceDeserializationContext.getSessionConfig(), serviceInstanceConfig.getServiceId(), serviceInstanceDeserializationContext.getTransaction());
		final String serviceInstanceName = ManagerFactory.getLocalizationManager().localize(serviceInstanceConfig.getName(), serviceConfig.getResource());
		
		ITransaction transaction = serviceInstanceDeserializationContext.getTransaction();

		// initialize progress bar
		serviceInstanceDeserializationContext.initializeInnerProgressBar(contentData.dataCount, "STR_IMPORTING_SERVICE_INSTANCE_DATA;" + serviceInstanceName);

		for (ContentSourceSerializableConfig sourceContentData : contentData.sources)
		{
			// deserialize content
			deserializeTableData(serviceInstanceDeserializationContext, sourceContentData.id, sourceContentData.tableData, false, transaction);
			
			if(sourceContentData.tableData instanceof FileTableSerializableConfig)
				deserializeAttachedFiles((FileTableSerializableConfig)sourceContentData.tableData, serviceInstanceDeserializationContext);
		}
		
		try
		{
			updateContentLocale(serviceInstanceDeserializationContext, contentData, transaction);
		}
		catch (SQLException e)
		{
			throw new UnexpectedException(e);
		}
		
		ITransactionObserver transactionObserver = new AbstractTransactionObserver()
		{
			@Override
			public void afterCommit() throws PortalException
			{
				ITransaction transaction = PortalTransactionFactory.createTransaction();
				try
				{
					transaction.begin();
					ManagerFactory.getFileManager().publicServiceInstanceFiles(serviceInstanceDeserializationContext.getSessionConfig(), serviceInstanceConfig.getId(), serviceInstanceConfig.getUsesPublicFileStorage(), transaction);
					transaction.commit();
				}
				finally
				{
					transaction.dispose();
				}
			}
		};
		transaction.addObserver(transactionObserver);
		
		// re-index the service instance's data after transaction commit
		ITransactionObserver reindexTransactionObserver = new AbstractTransactionObserver()
		{
			@Override
			public void afterCommit() throws PortalException
			{
				ITransaction transaction = PortalTransactionFactory.createTransaction();
				try
				{
					transaction.begin();
					
					// send information to the progress bar
					serviceInstanceDeserializationContext.logDebug("Starting reindexing for service instance '" + serviceInstanceConfig.getId() + "'");
					serviceInstanceDeserializationContext.sendProgressNotification("STR_REINDEXING_SERVICE_INSTANCE_DATA;" + serviceInstanceName, ProgressConfig.Type.INFO);
					
					// perform the re-indexing
					DouiIndexerFactory.getDouiIndexer().indexServiceInstance(
							serviceInstanceDeserializationContext.getSessionConfig(), 
							serviceInstanceConfig.getId(), transaction);
					serviceInstanceDeserializationContext.logDebug("Completed reindexing for service instance '" + serviceInstanceConfig.getId() + "'");
					
					transaction.commit();
				}
				finally
				{
					transaction.dispose();
				}
			}
		};
		transaction.addObserver(reindexTransactionObserver);
		
		// finalize progress bar
		serviceInstanceDeserializationContext.finalizeInnerProgressBar("STR_IMPORTING_SERVICE_INSTANCE_DATA_FINISHED;" + contentData.dataCount + ";" + serviceInstanceName);
}
	
	/**
	 * Detectes the files names that need to be deserialized and creates the files in the data directory.
	 * 
	 * @param fileDataNode
	 * @param serviceInstanceDeserializationContext
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void deserializeAttachedFiles(FileTableSerializableConfig fileList, ServiceInstanceDeserializationContext serviceInstanceDeserializationContext) throws PortalException
	{
		if (fileList == null)
			return;
		
		for (String filePath : fileList.fileNames)
		{
			serviceInstanceDeserializationContext.logDebug(" Adding FileCopyTransactionObserver: destinationPath:DATA\\" + filePath);
			FileCopyTransactionObserver fileCopyTransactionObserver = new FileCopyTransactionObserver(serviceInstanceDeserializationContext.getSessionConfig(), "ContentService/lum_files"+filePath, serviceInstanceDeserializationContext.getDataDirectory(), filePath, RelativePathType.Data, serviceInstanceDeserializationContext.getTransaction());
			serviceInstanceDeserializationContext.getTransaction().addObserver(fileCopyTransactionObserver);
		}
	}

	/**
	 * Reads the table name, column names and row data from the given xml node
	 * and deserializes the data into the appropriate table. 
	 * @param progressEnabled TODO
	 * @param transaction
	 * @param tableDataNode
	 * 
	 * @throws PortalException
	 * @since 4.0.11
	 */
	protected void deserializeTableData(final ServiceInstanceDeserializationContext serviceInstanceDeserializationContext, 
			final String sourceId, final TableSerializableConfig tableConfig, final boolean progressEnabled, 
			final ITransaction transaction) throws PortalException
	{
		if (tableConfig == null)
			return;

		if (tableConfig.tableName != null)
		{
			if ("lum_ContentAssociation".equalsIgnoreCase(tableConfig.tableName))
			{
				transaction.addObserver(new AbstractTransactionObserver()
				{
					private final String serviceInstanceId = serviceInstanceDeserializationContext.getServiceInstanceId();
					private final String serviceInstanceName = serviceInstanceDeserializationContext.getServiceInstanceName();
					
					@Override
					public boolean beforeCommit() throws PortalException
					{
						serviceInstanceDeserializationContext.setServiceInstanceId(serviceInstanceId);
						serviceInstanceDeserializationContext.setServiceInstanceName(serviceInstanceName);
						
						for (int fileNum = 1; fileNum <= tableConfig.numDataFiles; fileNum++)
						{
							deserializeTableDataFile(serviceInstanceDeserializationContext, 
									sourceId, tableConfig, fileNum, progressEnabled, transaction);
						}
						return true;
					}
				});
			}
			else
			{
				for (int fileNum = 1; fileNum <= tableConfig.numDataFiles; fileNum++)
				{
					deserializeTableDataFile(serviceInstanceDeserializationContext, sourceId, tableConfig, fileNum, progressEnabled, transaction);
				}
			}
		}
	}
	
	protected void deserializeTableDataFile(ServiceInstanceDeserializationContext serviceInstanceDeserializationContext, String sourceId, TableSerializableConfig tableConfig, int fileNum, boolean progressEnabled, ITransaction transaction) throws PortalException
	{
		StringBuilder queryPrefix = new StringBuilder();
		String tableName = tableConfig.tableName;
		queryPrefix.append("insert into "+tableName+" (");
		
		int count = 0;
		List<TableColumnSerializableConfig> columnNames = tableConfig.columnNames;
		for (TableColumnSerializableConfig column : columnNames)
		{
			queryPrefix.append(column.columnName);
			
			count++;
			
			if (count < columnNames.size())
				queryPrefix.append(",");
		}
		
		queryPrefix.append(") ");
		
		// read the content data
		String filePath = dataFileDirectoryPath+"/"+sourceId+"/"+tableName+"/"+fileNum+".xml";
		CachedTemporaryDataSerializableConfig tableData = ManagerFactoryInternal.getSerializationManager2().deserializeObject(CachedTemporaryDataSerializableConfig.class, serviceInstanceDeserializationContext.getDataDirectory(), filePath, null,
			serviceInstanceDeserializationContext.getSerializationSettings(), true);
		
		for (TableRowDataSerializableConfig curRow : tableData.cachedTemporaryData)
		{
			ArrayList<ColumnValue> values = new ArrayList<ColumnValue>();
			StringBuilder query = new StringBuilder();
			
			query.append(queryPrefix);

			query.append("values(");
			
			int colIndex = 0;
			List<Object> rowData = curRow.rowData;
			for (Object columnValue : rowData)
			{
				if (colIndex != 0)
					query.append(",");
				
				TableColumnSerializableConfig tableColumn = columnNames.get(colIndex);
				
				// assume null value for activeVersionId and publishedVersionId, 
				// as they are set later in updateContentLocale,
				// because of foreign keys
				if (columnValue == null || 
						("lum_ContentLocale".equalsIgnoreCase(tableName) &&
						("activeVersionId".equalsIgnoreCase(tableColumn.columnName) ||
						"publishedVersionId".equalsIgnoreCase(tableColumn.columnName))))
				{
					query.append("NULL");
				}
				else
				{
					query.append("?");
					ColumnValue value = new ColumnValue(columnValue, tableColumn.isClob);
					values.add(value);
				}
				
				colIndex++;
			}
			query.append(")");

			ITransactionJdbc daoTransactionJdbc = (ITransactionJdbc) transaction;
			Connection connection = daoTransactionJdbc.getConnection();

			try
			{
				PreparedStatement statement = connection.prepareStatement(query.toString());

				try
				{
					int i = 1;
					for (ColumnValue value : values)
					{
						if (value.isClob)
							Hibernate.TEXT.nullSafeSet(statement, value.value, i++);
						else
							statement.setObject(i++, value.value);
					}
					statement.executeUpdate();
				}
				finally
				{
					statement.close();
				}

				// update progress bar
				if (progressEnabled)
					serviceInstanceDeserializationContext.updateInnerProgressBar(1, null);
				
			}
			catch (SQLException e) 
			{
				// if during the insert of a content publication-service instance relationship
				// check if is due to inexistent service instance (dependency not resolved) 
				if("lum_ContentPubServiceInst".equals(tableName))
				{
					// gets the indexes of columns
					String serviceInstanceId = null;
					Object contentPublicationId = null;
					int index = 0;
					for(TableColumnSerializableConfig columnName:columnNames)
					{
						if("serviceInstanceId".equalsIgnoreCase(columnName.columnName))
							serviceInstanceId = (String)values.get(index).value;
						else if("contentPublicationId".equalsIgnoreCase(columnName.columnName))
							contentPublicationId = values.get(index).value;
						index++;
					}
					
					// verifies if the exception is expected
					try
					{
						ManagerFactory.getServiceInstanceManager().get(serviceInstanceId, transaction);
					}
					catch (PortalObjectNotFoundException e2)
					{
						serviceInstanceDeserializationContext.logDebug("Content publication external dependency error: contentPublicationId '"+
								contentPublicationId+"' of service instance '"+serviceInstanceDeserializationContext.getServiceInstanceName()+
								"' with serviceInstanceId '"+serviceInstanceDeserializationContext.getServiceInstanceId()+
								"' depends on serviceInstanceId '"+serviceInstanceId+"' that does not exist.");
						
						throw new PortalException("STR_SOME_CONFIG_COULD_NOT_BE_DESERIALIZED_DUE_TO_UNRESOLVED_DEPENDENCIES", e);
					}
				}

				throw new UnexpectedException(e);
			}
			
		}
	}
	
	/**
	 * Updates the activeVersionId and publishedVersionId columns in the
	 * lum_ContentLocale table. This update is done separately, because
	 * null values are inserted in these columns due to foreign keys constraints.
	 * @param deserializationContext the deserialization context.
	 * @param contentData the data being deserialized.
	 * @param transaction the transaction for persistence access.
	 * @since 4.2.0
	 */
	private void updateContentLocale(ServiceInstanceDeserializationContext deserializationContext, 
			ContentSerializableConfig contentData, ITransaction transaction) throws SQLException, PortalException
	{
		ITransactionJdbc daoTransactionJdbc = (ITransactionJdbc) transaction;
		Connection connection = daoTransactionJdbc.getConnection();

		String queryUpdate = "update lum_ContentLocale set activeVersionId=?,publishedVersionId=? where id=?";
		PreparedStatement statement = connection.prepareStatement(queryUpdate);
		try
		{
			for (ContentSourceSerializableConfig sourceContentData : contentData.sources)
			{
				TableSerializableConfig tableConfig = sourceContentData.tableData;
				if ("lum_ContentLocale".equalsIgnoreCase(tableConfig.tableName))
				{
					// get indices for columns that will be used
					int activeVersionColumnIndex = -1;
					int publishedVersionColumnIndex = -1;
					int contentLocaleIdColumnIndex = -1;
					for (int i=0; i<tableConfig.columnNames.size(); i++)
					{
						String columnName = tableConfig.columnNames.get(i).columnName;
						if ("activeVersionId".equalsIgnoreCase(columnName))
							activeVersionColumnIndex = i;
						else if ("publishedVersionId".equalsIgnoreCase(columnName))
							publishedVersionColumnIndex = i;
						else if ("id".equalsIgnoreCase(columnName))
							contentLocaleIdColumnIndex = i;
					}
					
					for (int fileNum = 1; fileNum <= tableConfig.numDataFiles; fileNum++)
					{
						// read the content data
						String filePath = dataFileDirectoryPath+"/"+sourceContentData.id+"/"+tableConfig.tableName+"/"+fileNum+".xml";
						CachedTemporaryDataSerializableConfig tableData = ManagerFactoryInternal.getSerializationManager2().deserializeObject(CachedTemporaryDataSerializableConfig.class, deserializationContext.getDataDirectory(), filePath, null,
								deserializationContext.getSerializationSettings(), true);

						// execute the update in the database
						for (TableRowDataSerializableConfig curRow : tableData.cachedTemporaryData)
						{
							statement.setString(1, (String)curRow.rowData.get(activeVersionColumnIndex));
							statement.setString(2, (String)curRow.rowData.get(publishedVersionColumnIndex));
							statement.setString(3, (String)curRow.rowData.get(contentLocaleIdColumnIndex));
							statement.executeUpdate();
						}
					}
				}
			}
		}
		finally
		{
			statement.close();
		}
	}

	// GENERAL

	/**
	 * Set up a serialization context, defining settings such as class aliases, field aliases,
	 * transient fields, omitted collections etc.
	 * 
	 * @param serviceInstanceBaseSerializationContext
	 */
	private void setupSerializationContext(ServiceInstanceBaseSerializationContext serviceInstanceBaseSerializationContext)
	{
		// setup serialization settings
		ContentManagerFactory.getWorkflowSerializationManager().setupSerializationContext(serviceInstanceBaseSerializationContext);
		
		if (serviceInstanceBaseSerializationContext.getIncludeContent())
		{
			Map<Class, String> classAliases = serviceInstanceBaseSerializationContext.getSerializationSettings().getClassAliases();
			classAliases.put(ContentSerializableConfig.class, "contentStructure");
			classAliases.put(ContentSourceSerializableConfig.class, "contentSource");
			classAliases.put(TableSerializableConfig.class, "contentTable");
			classAliases.put(TableColumnSerializableConfig.class, "contentColumn");
			classAliases.put(FileTableSerializableConfig.class, "contentFileTable");
			classAliases.put(CachedTemporaryDataSerializableConfig.class, "contentCachedData");
			classAliases.put(TableRowDataSerializableConfig.class, "contentRow");
			
			serviceInstanceBaseSerializationContext.getSerializationSettings().getTransientFieldsForClass(TableSerializableConfig.class).add("cachedData");
			serviceInstanceBaseSerializationContext.getSerializationSettings().getTransientFieldsForClass(ContentSourceSerializableConfig.class).add("sourceFromStatement");
		}
	}
}
